{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "sys.path.append(os.path.abspath(os.path.join('..')))\n",
    "from ch07_autograd.utils import Scalar, draw_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"215pt\" height=\"405pt\"\n",
       " viewBox=\"0.00 0.00 215.20 404.58\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 400.578)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-400.578 211.2,-400.578 211.2,4 -4,4\"/>\n",
       "<!-- 140231043894480backward -->\n",
       "<g id=\"node1\" class=\"node\"><title>140231043894480backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"0,-112.719 0,-171.641 76.2051,-171.641 76.2051,-112.719 0,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"38.1025\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.15</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"0,-152 76.2051,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"38.1025\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"0,-132.359 76.2051,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"38.1025\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">b</text>\n",
       "</g>\n",
       "<!-- 140231043894528backward -->\n",
       "<g id=\"node2\" class=\"node\"><title>140231043894528backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M106,-112.719C106,-112.719 158.205,-112.719 158.205,-112.719 164.205,-112.719 170.205,-118.719 170.205,-124.719 170.205,-124.719 170.205,-159.641 170.205,-159.641 170.205,-165.641 164.205,-171.641 158.205,-171.641 158.205,-171.641 106,-171.641 106,-171.641 100,-171.641 94,-165.641 94,-159.641 94,-159.641 94,-124.719 94,-124.719 94,-118.719 100,-112.719 106,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"132.103\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.15</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"94,-152 170.205,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"132.103\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=0.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"94,-132.359 170.205,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"132.103\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 140231043894576backward -->\n",
       "<g id=\"node3\" class=\"node\"><title>140231043894576backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"51,-0.5 51,-59.4219 127.205,-59.4219 127.205,-0.5 51,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"89.1025\" y=\"-47.4219\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.01</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"51,-39.7812 127.205,-39.7812 \"/>\n",
       "<text text-anchor=\"middle\" x=\"89.1025\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">value=5.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"51,-20.1406 127.205,-20.1406 \"/>\n",
       "<text text-anchor=\"middle\" x=\"89.1025\" y=\"-8.14062\" font-family=\"Menlo\" font-size=\"10.00\">w</text>\n",
       "</g>\n",
       "<!-- 140231043894528backward&#45;&gt;140231043894576backward -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>140231043894528backward&#45;&gt;140231043894576backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M120.917,-112.509C115.741,-99.2417 109.52,-83.2956 103.957,-69.0367\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"107.127,-67.5324 100.232,-59.4883 100.606,-70.0766 107.127,-67.5324\"/>\n",
       "<text text-anchor=\"middle\" x=\"130.96\" y=\"-83.0187\" font-family=\"Menlo\" font-size=\"14.00\">0.01</text>\n",
       "</g>\n",
       "<!-- 140231043894624backward -->\n",
       "<g id=\"node4\" class=\"node\"><title>140231043894624backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"176.103\" cy=\"-29.9609\" rx=\"31.1949\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"176.103\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">x=0.10</text>\n",
       "</g>\n",
       "<!-- 140231043894528backward&#45;&gt;140231043894624backward -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>140231043894528backward&#45;&gt;140231043894624backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M144.555,-112.446C147.096,-106.442 149.72,-100.138 152.103,-94.2188 158.418,-78.5274 165.222,-60.4737 169.967,-47.6801\"/>\n",
       "</g>\n",
       "<!-- 140231043894672backward -->\n",
       "<g id=\"node5\" class=\"node\"><title>140231043894672backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M40,-337.156C40,-337.156 92.2051,-337.156 92.2051,-337.156 98.2051,-337.156 104.205,-343.156 104.205,-349.156 104.205,-349.156 104.205,-384.078 104.205,-384.078 104.205,-390.078 98.2051,-396.078 92.2051,-396.078 92.2051,-396.078 40,-396.078 40,-396.078 34,-396.078 28,-390.078 28,-384.078 28,-384.078 28,-349.156 28,-349.156 28,-343.156 34,-337.156 40,-337.156\"/>\n",
       "<text text-anchor=\"middle\" x=\"66.1025\" y=\"-384.078\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"28,-376.438 104.205,-376.438 \"/>\n",
       "<text text-anchor=\"middle\" x=\"66.1025\" y=\"-364.438\" font-family=\"Menlo\" font-size=\"10.00\">value=0.82</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"28,-356.797 104.205,-356.797 \"/>\n",
       "<text text-anchor=\"middle\" x=\"66.1025\" y=\"-344.797\" font-family=\"Menlo\" font-size=\"10.00\">sigmoid</text>\n",
       "</g>\n",
       "<!-- 140231043894720backward -->\n",
       "<g id=\"node6\" class=\"node\"><title>140231043894720backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M40,-224.938C40,-224.938 92.2051,-224.938 92.2051,-224.938 98.2051,-224.938 104.205,-230.938 104.205,-236.938 104.205,-236.938 104.205,-271.859 104.205,-271.859 104.205,-277.859 98.2051,-283.859 92.2051,-283.859 92.2051,-283.859 40,-283.859 40,-283.859 34,-283.859 28,-277.859 28,-271.859 28,-271.859 28,-236.938 28,-236.938 28,-230.938 34,-224.938 40,-224.938\"/>\n",
       "<text text-anchor=\"middle\" x=\"66.1025\" y=\"-271.859\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.15</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"28,-264.219 104.205,-264.219 \"/>\n",
       "<text text-anchor=\"middle\" x=\"66.1025\" y=\"-252.219\" font-family=\"Menlo\" font-size=\"10.00\">value=1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"28,-244.578 104.205,-244.578 \"/>\n",
       "<text text-anchor=\"middle\" x=\"66.1025\" y=\"-232.578\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 140231043894672backward&#45;&gt;140231043894720backward -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>140231043894672backward&#45;&gt;140231043894720backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M66.1025,-336.946C66.1025,-323.934 66.1025,-308.345 66.1025,-294.299\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"69.6026,-293.926 66.1025,-283.926 62.6026,-293.926 69.6026,-293.926\"/>\n",
       "<text text-anchor=\"middle\" x=\"82.96\" y=\"-307.456\" font-family=\"Menlo\" font-size=\"14.00\">0.15</text>\n",
       "</g>\n",
       "<!-- 140231043894720backward&#45;&gt;140231043894480backward -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>140231043894720backward&#45;&gt;140231043894480backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M58.819,-224.727C55.481,-211.588 51.4755,-195.821 47.8799,-181.667\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"51.2042,-180.537 45.3496,-171.707 44.4197,-182.261 51.2042,-180.537\"/>\n",
       "<text text-anchor=\"middle\" x=\"70.96\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.15</text>\n",
       "</g>\n",
       "<!-- 140231043894720backward&#45;&gt;140231043894528backward -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>140231043894720backward&#45;&gt;140231043894528backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M83.271,-224.727C91.3682,-211.205 101.132,-194.9 109.794,-180.435\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"112.885,-182.085 115.02,-171.707 106.88,-178.488 112.885,-182.085\"/>\n",
       "<text text-anchor=\"middle\" x=\"119.96\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.15</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f8a158ecf40>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 为了图形简洁易懂，假设只有一个权重项参数\n",
    "## 当线性输出较小时，梯度不会溢出\n",
    "w = Scalar(5.0, label='w')\n",
    "b = Scalar(1.0, label='b')\n",
    "x = Scalar(0.1, label='x', requires_grad=False)\n",
    "h = w * x + b\n",
    "l = h.sigmoid()\n",
    "l.backward()\n",
    "draw_graph(l, 'backward')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"226pt\" height=\"405pt\"\n",
       " viewBox=\"0.00 0.00 225.79 404.58\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 400.578)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-400.578 221.792,-400.578 221.792,4 -4,4\"/>\n",
       "<!-- 140231043891792backward -->\n",
       "<g id=\"node1\" class=\"node\"><title>140231043891792backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"35.5662,-112.719 35.5662,-171.641 123.812,-171.641 123.812,-112.719 35.5662,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"79.6892\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"35.5662,-152 123.812,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"79.6892\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=100.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"35.5662,-132.359 123.812,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"79.6892\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 140231043893472backward -->\n",
       "<g id=\"node3\" class=\"node\"><title>140231043893472backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"34.6892\" cy=\"-29.9609\" rx=\"34.8795\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"34.6892\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">x=20.10</text>\n",
       "</g>\n",
       "<!-- 140231043891792backward&#45;&gt;140231043893472backward -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>140231043891792backward&#45;&gt;140231043893472backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M67.9835,-112.509C59.5981,-91.9703 48.5913,-65.0114 41.577,-47.8311\"/>\n",
       "</g>\n",
       "<!-- 140231043893136backward -->\n",
       "<g id=\"node5\" class=\"node\"><title>140231043893136backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"87.5867,-0.5 87.5867,-59.4219 163.792,-59.4219 163.792,-0.5 87.5867,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"125.689\" y=\"-47.4219\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"87.5867,-39.7812 163.792,-39.7812 \"/>\n",
       "<text text-anchor=\"middle\" x=\"125.689\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">value=5.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"87.5867,-20.1406 163.792,-20.1406 \"/>\n",
       "<text text-anchor=\"middle\" x=\"125.689\" y=\"-8.14062\" font-family=\"Menlo\" font-size=\"10.00\">w</text>\n",
       "</g>\n",
       "<!-- 140231043891792backward&#45;&gt;140231043893136backward -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>140231043891792backward&#45;&gt;140231043893136backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M91.6551,-112.509C97.1921,-99.2417 103.847,-83.2956 109.798,-69.0367\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"113.162,-70.0649 113.783,-59.4883 106.702,-67.3688 113.162,-70.0649\"/>\n",
       "<text text-anchor=\"middle\" x=\"122.547\" y=\"-83.0187\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231043893328backward -->\n",
       "<g id=\"node2\" class=\"node\"><title>140231043893328backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"141.587,-112.719 141.587,-171.641 217.792,-171.641 217.792,-112.719 141.587,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"179.689\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"141.587,-152 217.792,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"179.689\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"141.587,-132.359 217.792,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"179.689\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">b</text>\n",
       "</g>\n",
       "<!-- 140231043893616backward -->\n",
       "<g id=\"node4\" class=\"node\"><title>140231043893616backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"112.587,-337.156 112.587,-396.078 188.792,-396.078 188.792,-337.156 112.587,-337.156\"/>\n",
       "<text text-anchor=\"middle\" x=\"150.689\" y=\"-384.078\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"112.587,-376.438 188.792,-376.438 \"/>\n",
       "<text text-anchor=\"middle\" x=\"150.689\" y=\"-364.438\" font-family=\"Menlo\" font-size=\"10.00\">value=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"112.587,-356.797 188.792,-356.797 \"/>\n",
       "<text text-anchor=\"middle\" x=\"150.689\" y=\"-344.797\" font-family=\"Menlo\" font-size=\"10.00\">sigmoid</text>\n",
       "</g>\n",
       "<!-- 140231043893664backward -->\n",
       "<g id=\"node6\" class=\"node\"><title>140231043893664backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"106.566,-224.938 106.566,-283.859 194.812,-283.859 194.812,-224.938 106.566,-224.938\"/>\n",
       "<text text-anchor=\"middle\" x=\"150.689\" y=\"-271.859\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"106.566,-264.219 194.812,-264.219 \"/>\n",
       "<text text-anchor=\"middle\" x=\"150.689\" y=\"-252.219\" font-family=\"Menlo\" font-size=\"10.00\">value=101.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"106.566,-244.578 194.812,-244.578 \"/>\n",
       "<text text-anchor=\"middle\" x=\"150.689\" y=\"-232.578\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 140231043893616backward&#45;&gt;140231043893664backward -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>140231043893616backward&#45;&gt;140231043893664backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M150.689,-336.946C150.689,-323.934 150.689,-308.345 150.689,-294.299\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"154.189,-293.926 150.689,-283.926 147.189,-293.926 154.189,-293.926\"/>\n",
       "<text text-anchor=\"middle\" x=\"167.547\" y=\"-307.456\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231043893664backward&#45;&gt;140231043891792backward -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>140231043893664backward&#45;&gt;140231043891792backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M132.22,-224.727C123.51,-211.205 113.006,-194.9 103.688,-180.435\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"106.424,-178.218 98.0658,-171.707 100.539,-182.009 106.424,-178.218\"/>\n",
       "<text text-anchor=\"middle\" x=\"136.547\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231043893664backward&#45;&gt;140231043893328backward -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>140231043893664backward&#45;&gt;140231043893328backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M158.233,-224.727C161.69,-211.588 165.839,-195.821 169.563,-181.667\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"173.023,-182.269 172.183,-171.707 166.254,-180.487 173.023,-182.269\"/>\n",
       "<text text-anchor=\"middle\" x=\"184.547\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f8a158ecd30>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 当线性输出较大时，梯度溢出\n",
    "w = Scalar(5.0, label='w')\n",
    "b = Scalar(1.0, label='b')\n",
    "x = Scalar(20.1, label='x', requires_grad=False)\n",
    "h = w * x + b\n",
    "l = h.sigmoid()\n",
    "l.backward()\n",
    "draw_graph(l, 'backward')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"296pt\" height=\"517pt\"\n",
       " viewBox=\"0.00 0.00 296.24 516.80\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 512.797)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-512.797 292.236,-512.797 292.236,4 -4,4\"/>\n",
       "<!-- 140231043894768backward -->\n",
       "<g id=\"node1\" class=\"node\"><title>140231043894768backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"244.113\" cy=\"-29.9609\" rx=\"38.0635\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"244.113\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">x2=&#45;0.40</text>\n",
       "</g>\n",
       "<!-- 140231044055104backward -->\n",
       "<g id=\"node2\" class=\"node\"><title>140231044055104backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M130.01,-449.375C130.01,-449.375 182.215,-449.375 182.215,-449.375 188.215,-449.375 194.215,-455.375 194.215,-461.375 194.215,-461.375 194.215,-496.297 194.215,-496.297 194.215,-502.297 188.215,-508.297 182.215,-508.297 182.215,-508.297 130.01,-508.297 130.01,-508.297 124.01,-508.297 118.01,-502.297 118.01,-496.297 118.01,-496.297 118.01,-461.375 118.01,-461.375 118.01,-455.375 124.01,-449.375 130.01,-449.375\"/>\n",
       "<text text-anchor=\"middle\" x=\"156.113\" y=\"-496.297\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118.01,-488.656 194.215,-488.656 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.113\" y=\"-476.656\" font-family=\"Menlo\" font-size=\"10.00\">value=1.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"118.01,-469.016 194.215,-469.016 \"/>\n",
       "<text text-anchor=\"middle\" x=\"156.113\" y=\"-457.016\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 140231043894384backward -->\n",
       "<g id=\"node3\" class=\"node\"><title>140231043894384backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"67.0103,-337.156 67.0103,-396.078 143.215,-396.078 143.215,-337.156 67.0103,-337.156\"/>\n",
       "<text text-anchor=\"middle\" x=\"105.113\" y=\"-384.078\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"67.0103,-376.438 143.215,-376.438 \"/>\n",
       "<text text-anchor=\"middle\" x=\"105.113\" y=\"-364.438\" font-family=\"Menlo\" font-size=\"10.00\">value=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"67.0103,-356.797 143.215,-356.797 \"/>\n",
       "<text text-anchor=\"middle\" x=\"105.113\" y=\"-344.797\" font-family=\"Menlo\" font-size=\"10.00\">sigmoid</text>\n",
       "</g>\n",
       "<!-- 140231044055104backward&#45;&gt;140231043894384backward -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>140231044055104backward&#45;&gt;140231043894384backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M142.846,-449.165C136.648,-435.77 129.187,-419.645 122.541,-405.282\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"125.689,-403.75 118.313,-396.145 119.336,-406.69 125.689,-403.75\"/>\n",
       "<text text-anchor=\"middle\" x=\"150.97\" y=\"-419.675\" font-family=\"Menlo\" font-size=\"14.00\">1.00</text>\n",
       "</g>\n",
       "<!-- 140231043893184backward -->\n",
       "<g id=\"node10\" class=\"node\"><title>140231043893184backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M173.01,-337.156C173.01,-337.156 225.215,-337.156 225.215,-337.156 231.215,-337.156 237.215,-343.156 237.215,-349.156 237.215,-349.156 237.215,-384.078 237.215,-384.078 237.215,-390.078 231.215,-396.078 225.215,-396.078 225.215,-396.078 173.01,-396.078 173.01,-396.078 167.01,-396.078 161.01,-390.078 161.01,-384.078 161.01,-384.078 161.01,-349.156 161.01,-349.156 161.01,-343.156 167.01,-337.156 173.01,-337.156\"/>\n",
       "<text text-anchor=\"middle\" x=\"199.113\" y=\"-384.078\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"161.01,-376.438 237.215,-376.438 \"/>\n",
       "<text text-anchor=\"middle\" x=\"199.113\" y=\"-364.438\" font-family=\"Menlo\" font-size=\"10.00\">value=0.50</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"161.01,-356.797 237.215,-356.797 \"/>\n",
       "<text text-anchor=\"middle\" x=\"199.113\" y=\"-344.797\" font-family=\"Menlo\" font-size=\"10.00\">sigmoid</text>\n",
       "</g>\n",
       "<!-- 140231044055104backward&#45;&gt;140231043893184backward -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>140231044055104backward&#45;&gt;140231043893184backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M167.298,-449.165C172.474,-435.898 178.695,-419.952 184.258,-405.693\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"187.609,-406.733 187.983,-396.145 181.088,-404.189 187.609,-406.733\"/>\n",
       "<text text-anchor=\"middle\" x=\"197.97\" y=\"-419.675\" font-family=\"Menlo\" font-size=\"14.00\">1.00</text>\n",
       "</g>\n",
       "<!-- 140231043892032backward -->\n",
       "<g id=\"node8\" class=\"node\"><title>140231043892032backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"63,-224.938 63,-283.859 145.226,-283.859 145.226,-224.938 63,-224.938\"/>\n",
       "<text text-anchor=\"middle\" x=\"104.113\" y=\"-271.859\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"63,-264.219 145.226,-264.219 \"/>\n",
       "<text text-anchor=\"middle\" x=\"104.113\" y=\"-252.219\" font-family=\"Menlo\" font-size=\"10.00\">value=75.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"63,-244.578 145.226,-244.578 \"/>\n",
       "<text text-anchor=\"middle\" x=\"104.113\" y=\"-232.578\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 140231043894384backward&#45;&gt;140231043892032backward -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>140231043894384backward&#45;&gt;140231043892032backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M104.853,-336.946C104.735,-323.934 104.593,-308.345 104.466,-294.299\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"107.962,-293.894 104.372,-283.926 100.963,-293.957 107.962,-293.894\"/>\n",
       "<text text-anchor=\"middle\" x=\"121.97\" y=\"-307.456\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231043892416backward -->\n",
       "<g id=\"node4\" class=\"node\"><title>140231043892416backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"100,-112.719 100,-171.641 182.226,-171.641 182.226,-112.719 100,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.25</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"100,-152 182.226,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=20.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"100,-132.359 182.226,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">b</text>\n",
       "</g>\n",
       "<!-- 140231043892944backward -->\n",
       "<g id=\"node5\" class=\"node\"><title>140231043892944backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"100,-0.5 100,-59.4219 182.226,-59.4219 182.226,-0.5 100,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-47.4219\" font-family=\"Menlo\" font-size=\"10.00\">grad=&#45;0.10</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"100,-39.7812 182.226,-39.7812 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">value=50.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"100,-20.1406 182.226,-20.1406 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-8.14062\" font-family=\"Menlo\" font-size=\"10.00\">w</text>\n",
       "</g>\n",
       "<!-- 140231044055248backward -->\n",
       "<g id=\"node6\" class=\"node\"><title>140231044055248backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M175.01,-224.938C175.01,-224.938 227.215,-224.938 227.215,-224.938 233.215,-224.938 239.215,-230.938 239.215,-236.938 239.215,-236.938 239.215,-271.859 239.215,-271.859 239.215,-277.859 233.215,-283.859 227.215,-283.859 227.215,-283.859 175.01,-283.859 175.01,-283.859 169.01,-283.859 163.01,-277.859 163.01,-271.859 163.01,-271.859 163.01,-236.938 163.01,-236.938 163.01,-230.938 169.01,-224.938 175.01,-224.938\"/>\n",
       "<text text-anchor=\"middle\" x=\"201.113\" y=\"-271.859\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.25</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"163.01,-264.219 239.215,-264.219 \"/>\n",
       "<text text-anchor=\"middle\" x=\"201.113\" y=\"-252.219\" font-family=\"Menlo\" font-size=\"10.00\">value=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"163.01,-244.578 239.215,-244.578 \"/>\n",
       "<text text-anchor=\"middle\" x=\"201.113\" y=\"-232.578\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 140231044055248backward&#45;&gt;140231043892416backward -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>140231044055248backward&#45;&gt;140231043892416backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M185.505,-224.727C178.213,-211.333 169.435,-195.207 161.617,-180.845\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"164.498,-178.817 156.642,-171.707 158.35,-182.163 164.498,-178.817\"/>\n",
       "<text text-anchor=\"middle\" x=\"191.97\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.25</text>\n",
       "</g>\n",
       "<!-- 140231044058608backward -->\n",
       "<g id=\"node11\" class=\"node\"><title>140231044058608backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M211.99,-112.719C211.99,-112.719 276.236,-112.719 276.236,-112.719 282.236,-112.719 288.236,-118.719 288.236,-124.719 288.236,-124.719 288.236,-159.641 288.236,-159.641 288.236,-165.641 282.236,-171.641 276.236,-171.641 276.236,-171.641 211.99,-171.641 211.99,-171.641 205.99,-171.641 199.99,-165.641 199.99,-159.641 199.99,-159.641 199.99,-124.719 199.99,-124.719 199.99,-118.719 205.99,-112.719 211.99,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"244.113\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.25</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"199.99,-152 288.236,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"244.113\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=&#45;20.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"199.99,-132.359 288.236,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"244.113\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 140231044055248backward&#45;&gt;140231044058608backward -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>140231044055248backward&#45;&gt;140231044058608backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M212.298,-224.727C217.474,-211.46 223.695,-195.514 229.258,-181.255\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"232.609,-182.295 232.983,-171.707 226.088,-179.751 232.609,-182.295\"/>\n",
       "<text text-anchor=\"middle\" x=\"242.97\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.25</text>\n",
       "</g>\n",
       "<!-- 140231043891984backward -->\n",
       "<g id=\"node7\" class=\"node\"><title>140231043891984backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"41.1128\" cy=\"-29.9609\" rx=\"34.8795\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.1128\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">x1=1.10</text>\n",
       "</g>\n",
       "<!-- 140231043892032backward&#45;&gt;140231043892416backward -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>140231043892032backward&#45;&gt;140231043892416backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M113.738,-224.727C118.191,-211.46 123.544,-195.514 128.331,-181.255\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"131.672,-182.301 131.536,-171.707 125.036,-180.073 131.672,-182.301\"/>\n",
       "<text text-anchor=\"middle\" x=\"141.97\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231043892080backward -->\n",
       "<g id=\"node9\" class=\"node\"><title>140231043892080backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"0,-112.719 0,-171.641 82.2256,-171.641 82.2256,-112.719 0,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.1128\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"0,-152 82.2256,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"41.1128\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=55.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"0,-132.359 82.2256,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"41.1128\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 140231043892032backward&#45;&gt;140231043892080backward -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>140231043892032backward&#45;&gt;140231043892080backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M87.7247,-224.727C79.9956,-211.205 70.6756,-194.9 62.4074,-180.435\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"65.4199,-178.652 57.4188,-171.707 59.3426,-182.126 65.4199,-178.652\"/>\n",
       "<text text-anchor=\"middle\" x=\"93.9702\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231043892080backward&#45;&gt;140231043892944backward -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>140231043892080backward&#45;&gt;140231043892944backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M67.1256,-112.509C79.7413,-98.6039 95.027,-81.7561 108.422,-66.9923\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.103,-69.2462 115.23,-59.4883 105.919,-64.5426 111.103,-69.2462\"/>\n",
       "<text text-anchor=\"middle\" x=\"113.97\" y=\"-83.0187\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231043892080backward&#45;&gt;140231043891984backward -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>140231043892080backward&#45;&gt;140231043891984backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M41.1128,-112.509C41.1128,-92.0979 41.1128,-65.346 41.1128,-48.1524\"/>\n",
       "</g>\n",
       "<!-- 140231043893184backward&#45;&gt;140231044055248backward -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>140231043893184backward&#45;&gt;140231044055248backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M199.633,-336.946C199.869,-323.934 200.152,-308.345 200.407,-294.299\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"203.913,-293.988 200.595,-283.926 196.914,-293.861 203.913,-293.988\"/>\n",
       "<text text-anchor=\"middle\" x=\"217.97\" y=\"-307.456\" font-family=\"Menlo\" font-size=\"14.00\">0.25</text>\n",
       "</g>\n",
       "<!-- 140231044058608backward&#45;&gt;140231043894768backward -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>140231044058608backward&#45;&gt;140231043894768backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M244.113,-112.509C244.113,-92.0979 244.113,-65.346 244.113,-48.1524\"/>\n",
       "</g>\n",
       "<!-- 140231044058608backward&#45;&gt;140231043892944backward -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>140231044058608backward&#45;&gt;140231043892944backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M215.601,-112.567C209.721,-106.554 203.605,-100.216 197.969,-94.2188 189.835,-85.5636 181.199,-76.1022 173.18,-67.1958\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"175.602,-64.654 166.32,-59.5459 170.39,-69.3276 175.602,-64.654\"/>\n",
       "<text text-anchor=\"middle\" x=\"219.185\" y=\"-83.0187\" font-family=\"Menlo\" font-size=\"14.00\">&#45;0.10</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f8a15914250>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 不同数据点的反向传播是相互独立的\n",
    "w = Scalar(50.0, label='w')\n",
    "b = Scalar(20.0, label='b')\n",
    "x1 = Scalar(1.1, label='x1', requires_grad=False)\n",
    "x2 = Scalar(-0.4, label='x2', requires_grad=False)\n",
    "h1 = w * x1 + b\n",
    "l1 = h1.sigmoid()\n",
    "h2 = w * x2 + b\n",
    "l2 = h2.sigmoid()\n",
    "l = l1 + l2\n",
    "l.backward()\n",
    "draw_graph(l, 'backward')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.38.0 (20140413.2041)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"290pt\" height=\"517pt\"\n",
       " viewBox=\"0.00 0.00 290.23 516.80\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 512.797)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-512.797 286.226,-512.797 286.226,4 -4,4\"/>\n",
       "<!-- 140231044056592backward -->\n",
       "<g id=\"node1\" class=\"node\"><title>140231044056592backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"158,-224.938 158,-283.859 240.226,-283.859 240.226,-224.938 158,-224.938\"/>\n",
       "<text text-anchor=\"middle\" x=\"199.113\" y=\"-271.859\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"158,-264.219 240.226,-264.219 \"/>\n",
       "<text text-anchor=\"middle\" x=\"199.113\" y=\"-252.219\" font-family=\"Menlo\" font-size=\"10.00\">value=85.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"158,-244.578 240.226,-244.578 \"/>\n",
       "<text text-anchor=\"middle\" x=\"199.113\" y=\"-232.578\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 140231044056160backward -->\n",
       "<g id=\"node2\" class=\"node\"><title>140231044056160backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"200,-112.719 200,-171.641 282.226,-171.641 282.226,-112.719 200,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"241.113\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"200,-152 282.226,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"241.113\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=65.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"200,-132.359 282.226,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"241.113\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 140231044056592backward&#45;&gt;140231044056160backward -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>140231044056592backward&#45;&gt;140231044056160backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M210.038,-224.727C215.094,-211.46 221.17,-195.514 226.604,-181.255\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"229.952,-182.298 230.242,-171.707 223.411,-179.805 229.952,-182.298\"/>\n",
       "<text text-anchor=\"middle\" x=\"239.97\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231044056256backward -->\n",
       "<g id=\"node4\" class=\"node\"><title>140231044056256backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"100,-112.719 100,-171.641 182.226,-171.641 182.226,-112.719 100,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"100,-152 182.226,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=20.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"100,-132.359 182.226,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">b</text>\n",
       "</g>\n",
       "<!-- 140231044056592backward&#45;&gt;140231044056256backward -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>140231044056592backward&#45;&gt;140231044056256backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M184.025,-224.727C176.977,-211.333 168.491,-195.207 160.933,-180.845\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"163.879,-178.927 156.125,-171.707 157.684,-182.187 163.879,-178.927\"/>\n",
       "<text text-anchor=\"middle\" x=\"190.97\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231044056400backward -->\n",
       "<g id=\"node7\" class=\"node\"><title>140231044056400backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"241.113\" cy=\"-29.9609\" rx=\"34.8795\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"241.113\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">x3=1.30</text>\n",
       "</g>\n",
       "<!-- 140231044056160backward&#45;&gt;140231044056400backward -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>140231044056160backward&#45;&gt;140231044056400backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M241.113,-112.509C241.113,-92.0979 241.113,-65.346 241.113,-48.1524\"/>\n",
       "</g>\n",
       "<!-- 140231044056496backward -->\n",
       "<g id=\"node10\" class=\"node\"><title>140231044056496backward</title>\n",
       "<polygon fill=\"lightgreen\" stroke=\"black\" stroke-width=\"2\" points=\"100,-0.5 100,-59.4219 182.226,-59.4219 182.226,-0.5 100,-0.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-47.4219\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"100,-39.7812 182.226,-39.7812 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">value=50.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-width=\"2\" points=\"100,-20.1406 182.226,-20.1406 \"/>\n",
       "<text text-anchor=\"middle\" x=\"141.113\" y=\"-8.14062\" font-family=\"Menlo\" font-size=\"10.00\">w</text>\n",
       "</g>\n",
       "<!-- 140231044056160backward&#45;&gt;140231044056496backward -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>140231044056160backward&#45;&gt;140231044056496backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M215.1,-112.509C202.484,-98.6039 187.199,-81.7561 173.804,-66.9923\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"176.307,-64.5426 166.995,-59.4883 171.123,-69.2462 176.307,-64.5426\"/>\n",
       "<text text-anchor=\"middle\" x=\"213.97\" y=\"-83.0187\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231044057264backward -->\n",
       "<g id=\"node3\" class=\"node\"><title>140231044057264backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"58,-224.938 58,-283.859 140.226,-283.859 140.226,-224.938 58,-224.938\"/>\n",
       "<text text-anchor=\"middle\" x=\"99.1128\" y=\"-271.859\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"58,-264.219 140.226,-264.219 \"/>\n",
       "<text text-anchor=\"middle\" x=\"99.1128\" y=\"-252.219\" font-family=\"Menlo\" font-size=\"10.00\">value=90.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"58,-244.578 140.226,-244.578 \"/>\n",
       "<text text-anchor=\"middle\" x=\"99.1128\" y=\"-232.578\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 140231044057264backward&#45;&gt;140231044056256backward -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>140231044057264backward&#45;&gt;140231044056256backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M109.501,-224.682C113.473,-213.835 118.084,-201.396 122.398,-190.141 123.493,-187.283 124.637,-184.332 125.797,-181.366\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"129.101,-182.528 129.511,-171.941 122.588,-179.962 129.101,-182.528\"/>\n",
       "<text text-anchor=\"middle\" x=\"139.97\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231044055728backward -->\n",
       "<g id=\"node5\" class=\"node\"><title>140231044055728backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"0,-112.719 0,-171.641 82.2256,-171.641 82.2256,-112.719 0,-112.719\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.1128\" y=\"-159.641\" font-family=\"Menlo\" font-size=\"10.00\">grad=0.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"0,-152 82.2256,-152 \"/>\n",
       "<text text-anchor=\"middle\" x=\"41.1128\" y=\"-140\" font-family=\"Menlo\" font-size=\"10.00\">value=70.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"0,-132.359 82.2256,-132.359 \"/>\n",
       "<text text-anchor=\"middle\" x=\"41.1128\" y=\"-120.359\" font-family=\"Menlo\" font-size=\"10.00\">*</text>\n",
       "</g>\n",
       "<!-- 140231044057264backward&#45;&gt;140231044055728backward -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>140231044057264backward&#45;&gt;140231044055728backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M84.0254,-224.727C76.9768,-211.333 68.4911,-195.207 60.933,-180.845\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"63.8789,-178.927 56.1247,-171.707 57.6842,-182.187 63.8789,-178.927\"/>\n",
       "<text text-anchor=\"middle\" x=\"90.9702\" y=\"-195.238\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231044056352backward -->\n",
       "<g id=\"node6\" class=\"node\"><title>140231044056352backward</title>\n",
       "<ellipse fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" cx=\"41.1128\" cy=\"-29.9609\" rx=\"34.8795\" ry=\"18\"/>\n",
       "<text text-anchor=\"middle\" x=\"41.1128\" y=\"-27.7812\" font-family=\"Menlo\" font-size=\"10.00\">x4=1.40</text>\n",
       "</g>\n",
       "<!-- 140231044055728backward&#45;&gt;140231044056352backward -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>140231044055728backward&#45;&gt;140231044056352backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M41.1128,-112.509C41.1128,-92.0979 41.1128,-65.346 41.1128,-48.1524\"/>\n",
       "</g>\n",
       "<!-- 140231044055728backward&#45;&gt;140231044056496backward -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>140231044055728backward&#45;&gt;140231044056496backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M67.1256,-112.509C79.7413,-98.6039 95.027,-81.7561 108.422,-66.9923\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"111.103,-69.2462 115.23,-59.4883 105.919,-64.5426 111.103,-69.2462\"/>\n",
       "<text text-anchor=\"middle\" x=\"113.97\" y=\"-83.0187\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231044056928backward -->\n",
       "<g id=\"node8\" class=\"node\"><title>140231044056928backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"65.0103,-337.156 65.0103,-396.078 141.215,-396.078 141.215,-337.156 65.0103,-337.156\"/>\n",
       "<text text-anchor=\"middle\" x=\"103.113\" y=\"-384.078\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"65.0103,-376.438 141.215,-376.438 \"/>\n",
       "<text text-anchor=\"middle\" x=\"103.113\" y=\"-364.438\" font-family=\"Menlo\" font-size=\"10.00\">value=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"65.0103,-356.797 141.215,-356.797 \"/>\n",
       "<text text-anchor=\"middle\" x=\"103.113\" y=\"-344.797\" font-family=\"Menlo\" font-size=\"10.00\">sigmoid</text>\n",
       "</g>\n",
       "<!-- 140231044056928backward&#45;&gt;140231044057264backward -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>140231044056928backward&#45;&gt;140231044057264backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M102.072,-336.946C101.6,-323.934 101.034,-308.345 100.525,-294.299\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"104.009,-293.792 100.148,-283.926 97.0132,-294.046 104.009,-293.792\"/>\n",
       "<text text-anchor=\"middle\" x=\"118.97\" y=\"-307.456\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231044056448backward -->\n",
       "<g id=\"node9\" class=\"node\"><title>140231044056448backward</title>\n",
       "<polygon fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"160.01,-337.156 160.01,-396.078 236.215,-396.078 236.215,-337.156 160.01,-337.156\"/>\n",
       "<text text-anchor=\"middle\" x=\"198.113\" y=\"-384.078\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"160.01,-376.438 236.215,-376.438 \"/>\n",
       "<text text-anchor=\"middle\" x=\"198.113\" y=\"-364.438\" font-family=\"Menlo\" font-size=\"10.00\">value=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" stroke-dasharray=\"5,2\" points=\"160.01,-356.797 236.215,-356.797 \"/>\n",
       "<text text-anchor=\"middle\" x=\"198.113\" y=\"-344.797\" font-family=\"Menlo\" font-size=\"10.00\">sigmoid</text>\n",
       "</g>\n",
       "<!-- 140231044056448backward&#45;&gt;140231044056592backward -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>140231044056448backward&#45;&gt;140231044056592backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" stroke-dasharray=\"5,2\" d=\"M198.373,-336.946C198.491,-323.934 198.632,-308.345 198.76,-294.299\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"202.263,-293.957 198.854,-283.926 195.263,-293.894 202.263,-293.957\"/>\n",
       "<text text-anchor=\"middle\" x=\"215.97\" y=\"-307.456\" font-family=\"Menlo\" font-size=\"14.00\">0.00</text>\n",
       "</g>\n",
       "<!-- 140231044057072backward -->\n",
       "<g id=\"node11\" class=\"node\"><title>140231044057072backward</title>\n",
       "<path fill=\"#f0f0f0\" stroke=\"black\" d=\"M129.01,-449.375C129.01,-449.375 181.215,-449.375 181.215,-449.375 187.215,-449.375 193.215,-455.375 193.215,-461.375 193.215,-461.375 193.215,-496.297 193.215,-496.297 193.215,-502.297 187.215,-508.297 181.215,-508.297 181.215,-508.297 129.01,-508.297 129.01,-508.297 123.01,-508.297 117.01,-502.297 117.01,-496.297 117.01,-496.297 117.01,-461.375 117.01,-461.375 117.01,-455.375 123.01,-449.375 129.01,-449.375\"/>\n",
       "<text text-anchor=\"middle\" x=\"155.113\" y=\"-496.297\" font-family=\"Menlo\" font-size=\"10.00\">grad=1.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"117.01,-488.656 193.215,-488.656 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155.113\" y=\"-476.656\" font-family=\"Menlo\" font-size=\"10.00\">value=2.00</text>\n",
       "<polyline fill=\"none\" stroke=\"black\" points=\"117.01,-469.016 193.215,-469.016 \"/>\n",
       "<text text-anchor=\"middle\" x=\"155.113\" y=\"-457.016\" font-family=\"Menlo\" font-size=\"10.00\">+</text>\n",
       "</g>\n",
       "<!-- 140231044057072backward&#45;&gt;140231044056928backward -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>140231044057072backward&#45;&gt;140231044056928backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M141.586,-449.165C135.267,-435.77 127.659,-419.645 120.883,-405.282\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"124.004,-403.695 116.572,-396.145 117.673,-406.682 124.004,-403.695\"/>\n",
       "<text text-anchor=\"middle\" x=\"149.97\" y=\"-419.675\" font-family=\"Menlo\" font-size=\"14.00\">1.00</text>\n",
       "</g>\n",
       "<!-- 140231044057072backward&#45;&gt;140231044056448backward -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>140231044057072backward&#45;&gt;140231044056448backward</title>\n",
       "<path fill=\"none\" stroke=\"deepskyblue\" d=\"M166.298,-449.165C171.474,-435.898 177.695,-419.952 183.258,-405.693\"/>\n",
       "<polygon fill=\"deepskyblue\" stroke=\"deepskyblue\" points=\"186.609,-406.733 186.983,-396.145 180.088,-404.189 186.609,-406.733\"/>\n",
       "<text text-anchor=\"middle\" x=\"196.97\" y=\"-419.675\" font-family=\"Menlo\" font-size=\"14.00\">1.00</text>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.graphs.Digraph at 0x7f8a15914820>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 当对于所有数据，相应的线性输出都较大时，该神经元相当于坏死掉\n",
    "w = Scalar(50.0, label='w')\n",
    "b = Scalar(20.0, label='b')\n",
    "x3 = Scalar(1.3, label='x3', requires_grad=False)\n",
    "x4 = Scalar(1.4, label='x4', requires_grad=False)\n",
    "h3 = w * x3 + b\n",
    "l3 = h3.sigmoid()\n",
    "h4 = w * x4 + b\n",
    "l4 = h4.sigmoid()\n",
    "l = l3 + l4\n",
    "l.backward()\n",
    "draw_graph(l, 'backward')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
